{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Cheng\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from tqdm import tqdm\n",
    "import re\n",
    "import pickle\n",
    "import jieba\n",
    "\n",
    "re_apos = re.compile(r\"(\\w)'s\\b\")         # make 's a separate word\n",
    "re_mw_punc = re.compile(r\"(\\w[’'])(\\w)\")  # other ' in a word creates 2 words\n",
    "re_punc = re.compile(\"([\\\"().,;:/_?!—])\") # add spaces around punctuation\n",
    "re_mult_space = re.compile(r\"  *\")        # replace multiple spaces with just one\n",
    "\n",
    "def simple_toks(sent):\n",
    "    sent = re_apos.sub(r\"\\1 's\", sent)\n",
    "    sent = re_mw_punc.sub(r\"\\1 \\2\", sent)\n",
    "    sent = re_punc.sub(r\" \\1 \", sent).replace('-', ' ')\n",
    "    sent = re_mult_space.sub(' ', sent)\n",
    "    return sent.lower().split()\n",
    "\n",
    "\n",
    "class MySentences(object):\n",
    "    def __init__(self, dirname):\n",
    "        self.dirname = dirname\n",
    " \n",
    "    def __iter__(self):\n",
    "        for line in tqdm(open(self.dirname)):\n",
    "            yield simple_toks(line)\n",
    "            \n",
    "class MyChineseSentences(object):\n",
    "    def __init__(self, dirname):\n",
    "        self.dirname = dirname\n",
    " \n",
    "    def __iter__(self):\n",
    "        for line in tqdm(open(self.dirname)):\n",
    "            yield jieba.cut(line)            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='data/training/'\n",
    "\n",
    "fname=path+'train'\n",
    "en_fname = fname+'.en'\n",
    "zh_fname = fname+'.zh'\n",
    "\n",
    "def train_save_wordvec_export(lan ='en'):\n",
    "    output = 'input/'+lan\n",
    "    if lan == 'en':\n",
    "        corpus = MySentences(en_fname)\n",
    "    else:\n",
    "        corpus = MyChineseSentences(zh_fname)\n",
    "    model = Word2Vec(corpus, min_count=10, sg=1,workers=2,iter=50)    \n",
    "    print('model trained!')\n",
    "    vocabulary = model.wv.vocab\n",
    "    embeddings = []\n",
    "    for word in vocabulary:\n",
    "        embeddings.append(model[word])    \n",
    "    embeddings = np.array(embeddings, dtype=np.float32)\n",
    "    pickle.dump(vocabulary, open(output+'vocab.pkl', \"wb\"))\n",
    "    np.save(output+'word2vec.npy',embeddings)\n",
    "    print('word vector saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'gbk' codec can't decode byte 0x9f in position 453: illegal multibyte sequence",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-bc5cbb9039a1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_save_wordvec_export\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlan\u001b[0m \u001b[1;33m=\u001b[0m\u001b[1;34m'en'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mtrain_save_wordvec_export\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlan\u001b[0m \u001b[1;33m=\u001b[0m\u001b[1;34m'zh'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-9-95611977da4d>\u001b[0m in \u001b[0;36mtrain_save_wordvec_export\u001b[1;34m(lan)\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m         \u001b[0mcorpus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMyChineseSentences\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzh_fname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWord2Vec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_count\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msg\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0miter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'model trained!'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[0mvocabulary\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\gensim\\models\\word2vec.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, sentences, size, alpha, window, min_count, max_vocab_size, sample, seed, workers, min_alpha, sg, hs, negative, cbow_mean, hashfxn, iter, null_word, trim_rule, sorted_vocab, batch_words, compute_loss)\u001b[0m\n\u001b[0;32m    501\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mGeneratorType\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    502\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"You can't pass a generator as the sentences argument. Try an iterator.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 503\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuild_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrim_rule\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrim_rule\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    504\u001b[0m             self.train(sentences, total_examples=self.corpus_count, epochs=self.iter,\n\u001b[0;32m    505\u001b[0m                        start_alpha=self.alpha, end_alpha=self.min_alpha)\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\gensim\\models\\word2vec.py\u001b[0m in \u001b[0;36mbuild_vocab\u001b[1;34m(self, sentences, keep_raw_vocab, trim_rule, progress_per, update)\u001b[0m\n\u001b[0;32m    575\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    576\u001b[0m         \"\"\"\n\u001b[1;32m--> 577\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprogress_per\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprogress_per\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrim_rule\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrim_rule\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# initial survey\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    578\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscale_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkeep_raw_vocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkeep_raw_vocab\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrim_rule\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrim_rule\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mupdate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# trim by min_count & precalculate downsampling\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    579\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfinalize_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# build tables & arrays\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\gensim\\models\\word2vec.py\u001b[0m in \u001b[0;36mscan_vocab\u001b[1;34m(self, sentences, progress_per, trim_rule)\u001b[0m\n\u001b[0;32m    587\u001b[0m         \u001b[0mvocab\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdefaultdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m         \u001b[0mchecked_string_types\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 589\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0msentence_no\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msentence\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    590\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mchecked_string_types\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    591\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstring_types\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-1-993ae902a0f1>\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdirname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m             \u001b[1;32myield\u001b[0m \u001b[0msimple_toks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tqdm\\_tqdm.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    870\u001b[0m \"\"\", fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[0;32m    871\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 872\u001b[1;33m             \u001b[1;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[1;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    873\u001b[0m                 \u001b[1;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    874\u001b[0m                 \u001b[1;31m# Update and print the progressbar.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m: 'gbk' codec can't decode byte 0x9f in position 453: illegal multibyte sequence"
     ]
    }
   ],
   "source": [
    "train_save_wordvec_export(lan ='en')\n",
    "train_save_wordvec_export(lan ='zh')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
